%  LaTeX support: latex@mdpi.com 
%  In case you need support, please attach all files that are necessary for compiling as well as the log file, and specify the details of your LaTeX setup (which operating system and LaTeX version / tools you are using).

%=================================================================
\documentclass[journal,article,submit,moreauthors,pdftex]{Definitions/mdpi} 

% If you would like to post an early version of this manuscript as a preprint, you may use preprint as the journal and change 'submit' to 'accept'. The document class line would be, e.g., \documentclass[preprints,article,accept,moreauthors,pdftex]{mdpi}. This is especially recommended for submission to arXiv, where line numbers should be removed before posting. For preprints.org, the editorial staff will make this change immediately prior to posting.

%--------------------
% Class Options:
%--------------------
%----------
% journal
%----------
% Choose between the following MDPI journals:
% acoustics, actuators, addictions, admsci, aerospace, agriculture, agriengineering, agronomy, ai, algorithms, animals, antibiotics, antibodies, antioxidants, applmech, applsci, arts, asc, asi, atmosphere, atoms, axioms, batteries, bdcc, behavsci , beverages, bioengineering, biology, biomedicines, biomimetics, biomolecules, biosensors, brainsci , buildings, cancers, carbon , catalysts, cells, ceramics, challenges, chemengineering, chemistry, chemosensors, children, civileng, cleantechnol, climate, clockssleep, cmd, coatings, colloids, computation, computers, condensedmatter, cosmetics, cryptography, crystals, dairy, data, dentistry, designs , diagnostics, diseases, diversity, drones, econometrics, economies, education, ejbc, ejihpe, electrochem, electronics, endocrines, energies, entropy, environments, epigenomes, est, fermentation, fibers, fire, fishes, fluids, foods, forecasting, forests, fractalfract, futureinternet, futurephys, galaxies, games, gastrointestdisord, gels, genealogy, genes, geohazards, geosciences, geriatrics, hazardousmatters, healthcare, hearts, heritage, highthroughput, horticulturae, humanities, hydrology, ijerph, ijfs, ijgi, ijms, ijtpp, informatics, information, infrastructures, inorganics, insects, instruments, inventions, iot, j, jcdd, jce, jcm, jcp, jcs, jdb, jfb, jfmk, jimaging, jintelligence, jlpea, jmmp, jmse, jne, jnt, jof, joitmc, jpm, jrfm, jsan, land, languages, laws, life, literature, logistics, lubricants, machines, magnetochemistry, make, marinedrugs, materials, mathematics, mca, medicina, medicines, medsci, membranes, metabolites, metals, microarrays, micromachines, microorganisms, minerals, modelling, molbank, molecules, mps, mti, nanomaterials, ncrna, ijns, neurosci, neuroglia, nitrogen, notspecified, nutrients, oceans, ohbm, optics, particles, pathogens, pharmaceuticals, pharmaceutics, pharmacy, philosophies, photonics, physics, plants, plasma, pollutants, polymers, polysaccharides, preprints , proceedings, processes, prosthesis, proteomes, psych, publications, quantumrep, quaternary, qubs, reactions, recycling, religions, remotesensing, reprodmed, reports, resources, risks, robotics, safety, sci, scipharm, sensors, separations, sexes, signals, sinusitis, smartcities, sna, societies, socsci, soilsystems, sports, standards, stats, surfaces, surgeries, sustainability, sustainableworld, symmetry, systems, technologies, telecom, test, tourismhosp, toxics, toxins, transplantology, tropicalmed, universe, urbansci, vaccines, vehicles, vetsci, vibration, viruses, vision, water, wem, wevj

%---------
% article
%---------
% The default type of manuscript is "article", but can be replaced by: 
% abstract, addendum, article, benchmark, book, bookreview, briefreport, casereport, changes, comment, commentary, communication, conceptpaper, conferenceproceedings, correction, conferencereport, expressionofconcern, extendedabstract, meetingreport, creative, datadescriptor, discussion, editorial, essay, erratum, hypothesis, interestingimages, letter, meetingreport, newbookreceived, obituary, opinion, projectreport, reply, retraction, review, perspective, protocol, shortnote, supfile, technicalnote, viewpoint
% supfile = supplementary materials

%----------
% submit
%----------
% The class option "submit" will be changed to "accept" by the Editorial Office when the paper is accepted. This will only make changes to the frontpage (e.g., the logo of the journal will get visible), the headings, and the copyright information. Also, line numbering will be removed. Journal info and pagination for accepted papers will also be assigned by the Editorial Office.

%------------------
% moreauthors
%------------------
% If there is only one author the class option oneauthor should be used. Otherwise use the class option moreauthors.

%---------
% pdftex
%---------
% The option pdftex is for use with pdfLaTeX. If eps figures are used, remove the option pdftex and use LaTeX and dvi2pdf.

%=================================================================
\firstpage{1} 
\makeatletter 
\setcounter{page}{\@firstpage} 
\makeatother
\pubvolume{xx}
\issuenum{1}
\articlenumber{5}
\pubyear{2020}
\copyrightyear{2020}
%\externaleditor{Academic Editor: name}
\history{Received: date; Accepted: date; Published: date}
%\updates{yes} % If there is an update available, un-comment this line

%% MDPI internal command: uncomment if new journal that already uses continuous page numbers 
%\continuouspages{yes}

%------------------------------------------------------------------
% The following line should be uncommented if the LaTeX file is uploaded to arXiv.org
%\pdfoutput=1

%=================================================================
% Add packages and commands here. The following packages are loaded in our class file: fontenc, inputenc, calc, indentfirst, fancyhdr, graphicx,epstopdf, lastpage, ifthen, lineno, float, amsmath, setspace, enumitem, mathpazo, booktabs, titlesec, etoolbox, tabto, xcolor, soul, multirow, microtype, tikz, totcount, amsthm, hyphenat, natbib, hyperref, footmisc, url, geometry, newfloat, caption
%\usepackage{CJKutf8}
\usepackage{listings}
%=================================================================
%% Please use the following mathematics environments: Theorem, Lemma, Corollary, Proposition, Characterization, Property, Problem, Example, ExamplesandDefinitions, Hypothesis, Remark, Definition, Notation, Assumption
%% For proofs, please use the proof environment (the amsthm package is loaded by the MDPI class).

%=================================================================
% Full title of the paper (Capitalized)
\Title{基于VGG16的CIFAR10数据集分类实验报告}

% Author Orchid ID: enter ID or remove command
%\newcommand{\orcidauthorA}{0000-0000-000-000X} % Add \orcidA{} behind the author's name
%\newcommand{\orcidauthorB}{0000-0000-000-000X} % Add \orcidB{} behind the author's name

% Authors, for the paper (add full first names)
\Author{凌建 $^{1}$}

% Authors, for metadata in PDF
\AuthorNames{凌建}

% Affiliations / Addresses (Add [1] after \address if there is only one affiliation.)
\address{%
$^{1}$ \quad 凌建;}

% Contact information of the corresponding author
%\corres{Correspondence: e-mail@e-mail.com; Tel.: (optional; include country code; if there are multiple corresponding authors, add author initials) +xx-xxxx-xxx-xxxx (F.L.)}

% Current address and/or shared authorship
%\firstnote{Current address: Affiliation 3} 
%\secondnote{These authors contributed equally to this work.}
% The commands \thirdnote{} till \eighthnote{} are available for further notes

%\simplesumm{} % Simple summary

%\conference{} % An extended version of a conference paper

% Abstract (Do not insert blank lines, i.e. \\) 
\abstract{本次深度学习实验报告数据集采用CIFAR10数据集，基础模型采用VGG16。使用pytorch复现VGG16，并在复现的基础上对学习率、全连接层、权重衰退进行修改实验。通过实验结果看，在基础模型上叠加使用优化器学习率衰减配置得到得实验效果最好，在测试数据集上准确率达到0.86。}

% Keywords
\keyword{CNN;VGG16;卷积;归一化;梯度爆炸;梯度消失}

% The fields PACS, MSC, and JEL may be left empty or commented out if not applicable
%\PACS{J0101}
%\MSC{}
%\JEL{}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Only for the journal Diversity
%\LSID{\url{http://}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Only for the journal Applied Sciences:
%\featuredapplication{Authors are encouraged to provide a concise description of the specific application or a potential application of the work. This section is not mandatory.}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Only for the journal Data:
%\dataset{DOI number or link to the deposited data set in cases where the data set is published or set to be published separately. If the data set is submitted and will be published as a supplement to this paper in the journal Data, this field will be filled by the editors of the journal. In this case, please make sure to submit the data set as a supplement when entering your manuscript into our manuscript editorial system.}

%\datasetlicense{license under which the data set is made available (CC0, CC-BY, CC-BY-SA, CC-BY-NC, etc.)}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Only for the journal Toxins
%\keycontribution{The breakthroughs or highlights of the manuscript. Authors can write one or two sentences to describe the most important part of the paper.}
% 去掉pdf的行号
\preto{\abstractkeywords}{\nolinenumbers}
% 中文支持
\usepackage[UTF8]{ctex}
%\setcounter{secnumdepth}{4}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{引言}
按深度学习结课报告要求，选择VGG16作为结课实验报告的基础模型，使用pytorch复现VGG16。由于使用CIFAR10数据集，所以在复现的模型基础上需要针对全连接层的输入层和输出层进行修改。在修改的VGG16模型上分别对学习率、增减全连接层、优化器权重衰退修改后实验，得到各个实验结果。此外，在本次实验中复现VGG16时遇到梯度爆炸和梯度消失问题，通过在卷积层后添加批归一化处理后解决问题。通过本次深度学习实验，加强了对卷积神经网络的学习。

在本次实验中，使用了CIFAR10数据集。该数据集包含60000张32x32的彩色图片，每张图片分为三个通道：R通道、G通道和B通道。在这60000个样本中，有50000个用于模型训练，其余10000个用于评估模型在该数据集上达到的准确率。

本次实验的平台为kaggle。Kaggle是一个数据建模和数据分析竞赛平台，它每周免费提供30小时的GPU训练时间，极大方便了无本地显卡资源的深度学习。

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{CNN}
卷积神经网络（convolutional neural network，CNN）是一类强大的、为处理图像数据而设计的神经网络。 在计算机视觉领域中，基于卷积神经网络的模型已经成为主流\cite{d2l}。卷积神经网络主要包含两类网络：卷积网络和全连接网络。卷积网络主要用于提取图像的特征，故也称为特征提取（学习）网络；全连接网络则根据提取的特征对样本进行分类，也常称分类网络\cite{dl_thoery}。

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{卷积层}
卷积（Convolution）是卷积神经网络中核心的操作，其主要作用是提取图像的局部特征，如边沿特征、纹理特征以及高层语义特征等\cite{dl_thoery}。

在计算机视觉任务中，卷积神经网络之所以能够展现优异的性能，主要归功于卷积神经网络中的卷积层。卷积层具有强大的特征提取能力。卷积层的特征提取是通过卷积层中的卷积核实现的，该过程是通过卷积和在输入图像上滑动并做卷积运算完成的\cite{2023-china-ai-wp}。卷积操作如图\ref{fig:conv}所示，图中是一个3x3的卷积核（灰色阴影部分）,填充1。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/white\_paper\_conv}
	\caption{卷积}
	\caption*{来源:https//github.com/vdumoulin/conv\_arithmetic/blob/master/gif/padding\_strides.gif}
	\label{fig:conv}
\end{figure}

\subsection{激活函数}
在卷积层之后会加入一个激活函数，通过引入非线性元素来提高网络的表达能力。浅层的网络模型可以使用sigmoid函数；ReLU函数常用于深层网络模型；循环神经网络则使用tanh函数。在卷积神经网络中尽量避免使用sigmoid函数和tanh函数，因为会造成梯度消失，应首选ReLU函数作为激活函数\cite{2023-china-ai-wp}。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/sigmoid}
	\caption{sigmoid}
	\caption*{来源:https://www.aitude.com/comparison-of-sigmoid-tanh-and-relu-activation-functions/}
	\label{fig:sigmoid}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.3]{images/tanh}
	\caption{tanh}
	\caption*{来源:https://www.aitude.com/comparison-of-sigmoid-tanh-and-relu-activation-functions/}
	\label{fig:tanh}
\end{figure}

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.3]{images/relu}
	\caption{relu}
	\caption*{来源:https://zh.wikipedia.org/wiki/线性整流函数}
	\label{fig:relu}
\end{figure}

\subsection{池化层}
池化操作时将某一位置相邻输出的总体统计特征作为此位置的输出。常见的池化操作有最大池化（Max-Pooling）和平均值池化（Mean-Pooling）。池化的过程本质上是对特征图采样的过程，在保留特征的同时减少参数的计算量，防止过拟合\cite{2023-china-ai-wp}。如图\ref{fig:pooling}所示，图中的池化核大小为2x2。
\begin{figure}[H]
	\centering
	\includegraphics{images/white\_paper\_pooling}
	\caption{池化}
	\caption*{来源:2023中国人工智能系列白皮书--深度学习}
	\label{fig:pooling}
\end{figure}

\subsection{全连接层}
全连接层是卷积神经网络的最后一层，可以看作多个线性层的堆叠，将学习到的特征表示映射到样本标记空间用于分类或者回归任务,因此全连接层的参数非常庞大\cite{2023-china-ai-wp}。全连接层如图\ref{fig:fc}所示。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.3]{images/fc}
	\caption{全连接层}
	\caption*{来源:https://www.kaggle.com/code/blurredmachine/vggnet-16-architecture-a-complete-guide}
	\label{fig:fc}
\end{figure}

\section{损失函数}
在深度学习中，损失函数（Loss Function）是用于衡量模型预测输出与实际目标之间差距的一种函数。它在训练模型的过程中起着至关重要的作用，因为它指导模型的参数更新以逐步减少误差，从而提升模型的性能。在训练神经网络时，损失函数的主要作用包括：
\begin{itemize}[leftmargin=*,labelsep=5.8mm,labelindent=\parindent]
	\item	误差计算：通过损失函数计算预测结果与真实结果之间的差异；
	\item	反向传播：通过损失函数计算的值通过反向传播算法，指导模型参数的更新，使模型逐步优化；
	\item	优化目标：在训练过程中通过最小化损失函数来调整参数，以此改进模型的预测性能。
\end{itemize}

\section{优化器}
在优化中，损失函数通常被称为优化问题的目标函数。按照传统惯例，大多数优化算法都关注的是最小化\cite{d2l}。优化器（Optimizer）是用于调整模型参数以最小化损失函数的算法。优化器在训练过程中至关重要，它通过反向传播算法计算梯度，并根据这些梯度更新模型参数，使模型逐渐逼近最优解（最小值）。梯度下降示意图如图\ref{fig:gd}所示，通过不断迭代寻找目标函数的最小值。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.7]{images/sgd}
	\caption{梯度下降示意图}
	\caption*{来源:https://zh-v2.d2l.ai/chapter\_optimization/gd.html}
	\label{fig:gd}
\end{figure}

\section{VGG16}
\unskip
\subsection{介绍}
VGG 的全称是Visual Geometry Group，是牛津大学科学与工程系的视觉几何组。VGG发布了从 VGG16 到 VGG19得一系列卷积网络模型，可应用于人脸识别和图像分类。VGG 研究卷积网络深度的初衷是了解卷积网络深度如何影响大规模图像分类和识别的准确性和精确度\cite{vgg}。

\subsection{模型架构}
VGG16是一个经典的深度学习模型，其具有16层深的卷积神经网络结构。VGG16的模型架构如图\ref{fig:vgg16}所示，它包含13个卷积层、5个最大池化层和3个线性层，采用ReLU作为激活函数，最后一层使用softmax激活函数进行分类，输入图片尺寸为224x224，卷积核为3x3，池化核为2x2。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16}
	\caption{VGG16}
	\caption*{来源:https://www.kaggle.com/code/blurredmachine/vggnet-16-architecture-a-complete-guide}
	\label{fig:vgg16}
\end{figure}   

\subsection{模型架构适配修改}
使用CIFAR10数据集进行分类任务时，由于输入图像为32x32，所以输入到全连接层的输入为512，故需要修改全连接层的输入为512。此外，vgg16默认输出1000分类，cifar10为10分类，需要对vgg16输出进行修改，修改后的网络架构如图\ref{fig:vgg16-cifar10}所示。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-cifar10}
	\caption{针对CIFAR10数据集修改后的VGG16网络模型}
	\label{fig:vgg16-cifar10}
\end{figure}

本次vgg16基础模型实验超参数设置情况：
\begin{itemize}[leftmargin=*,labelsep=5.8mm,labelindent=\parindent]
	\item	初始学习率为0.01；
	\item	batch\_size为128；
	\item	训练50轮；
	\item   优化器采用SGD；
	\item   损失函数采用交叉熵损失函数CrossEntropyLoss()。
\end{itemize}

本实验除了采用损失函数来观察训练效果，还采用了准确率来评估模型训练效果。准确率是分类任务中最常用的评估指标之一。它是指模型预测正确的样本数占总样本数的比例。准确率 = (预测正确的正例数 + 预测正确的负例数) / 总样本数

\section{实验结果}
\unskip
\subsection{基础模型效果}
在基础模型上训练结果如图\ref{fig:vgg16-base-cifar10-acc}所示，在测试数据集上的最高准确率达到0.8258。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-base-cifar10-acc}
	\caption{基础模型训练中准确度}
	\label{fig:vgg16-base-cifar10-acc}
\end{figure}

\subsection{在基础模型上增加全连接层}
在基础模型上增加全连接层训练结果如图\ref{fig:vgg16-base-cifar10-add-fc-acc}所示，在测试数据集上的最高准确率达到0.8189。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-base-cifar10-add-fc-acc}
	\caption{在基础模型上增加全连接层训练中准确度}
	\label{fig:vgg16-base-cifar10-add-fc-acc}
\end{figure}

\subsection{在基础模型上设置优化器weight\_decay}
在训练参数化机器学习模型时， 权重衰减（weight decay）是最广泛使用的正则化的技术之一， 它通常也被称为
正则化\cite{d2l}。权重衰减的主要作用是在损失函数中加入L2正则化项解决过拟合。
在基础模型上设置优化器weight\_decay为1e-4的训练结果如图\ref{fig:vgg16-base-cifar10-wieght-decay-acc}所示，在测试数据集上的最高准确率达到0.8143。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-base-cifar10-wieght-decay-acc}
	\caption{在基础模型上设置优化器weight\_decay训练中准确度}
	\label{fig:vgg16-base-cifar10-wieght-decay-acc}
\end{figure}

\subsection{在基础模型上设置学习率衰减}
在基础模型上设置学习率衰减，采用固定步长衰减。在学习率衰减实验中分为两类：一类是初始学习率为0.01；另一类是初始学习率为0.1。

\subsubsection{初始学习率为0.01}
在基础模型上设置学习率衰减训练结果如图\ref{fig:vgg16-base-cifar10-step-lr-from0.01-acc}所示，在测试数据集上的最高准确率达到0.8226。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-base-cifar10-step-lr-from0.01-acc}
	\caption{在基础模型上初始学习率为0.01并设置学习率衰减训练中的准确度}
	\label{fig:vgg16-base-cifar10-step-lr-from0.01-acc}
\end{figure}

\subsubsection{初始学习率为0.1}
在基础模型上修改初始学习率为0.1并设置学习率衰减的训练结果如图\ref{fig:vgg16-base-cifar10-step-lr-from0.1-acc}所示，在测试数据集上的最高准确率达到0.8683。
\begin{figure}[H]
	\centering
	\includegraphics[scale=0.5]{images/vgg16-base-cifar10-step-lr-from0.1-acc}
	\caption{在基础模型上初始学习率为0.1并设置学习率衰减训练中的准确度}
	\label{fig:vgg16-base-cifar10-step-lr-from0.1-acc}
\end{figure}

\subsection{实验对比}
通过对不同参数修改进行实验，从实验结果中发现设置初始学习率为0.1的固定步长衰减达到的准确率最高。所有实验效果如表\ref{table:model_modify_outcome}所示。

\begin{table}[H]
\centering
\begin{tabular}{ccc}
\toprule
\textbf{模型}	& \textbf{准确度}\\
\midrule
基础模型		& 0.8258\\
在基础模型上增加全连接层		& 0.8189\\
在基础模型上设置优化器weight\_decay & 0.8143\\
在基础模型上初始学习率为0.01固定步长衰减  	& 0.8226\\
在基础模型上初始学习率为0.1固定步长衰减	& 0.8683\\
\bottomrule
\end{tabular}
\caption{模型修改对比}
\label{table:model_modify_outcome}
\end{table}

\section{总结}
在本次实验中第一次用pytorch搭建VGG16模型时，由于经验不足，导致在最初搭建出来的模型训练时出现loss值不收敛，并且在训练轮数增多后出现loss值和准确度变为NaN。通过查阅资料后发现该现象是梯度爆炸和梯度消失，需要在卷积层后激活函数ReLU前增加归一化处理，重新修改模型后，问题得到解决。通过本次实验加深了对深度学习的认识，

在基于VGG16的CIFAR10数据集分类实验中，发现基于学习率衰减策略训练的结果最好，从折线图看基于学习率衰减策略训练在第10轮就达到最优值，学习速度快，避免了固定学习率学习需要一开始设置为一个较小值，导致学习速度缓慢。

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% optional
\appendixtitles{no} % Leave argument "no" if all appendix headings stay EMPTY (then no dot is printed after "Appendix A"). If the appendix sections contain a heading then change the argument to "yes".
\appendix
\section{实验代码}
各个实验代码链接：https://gitee.com/LingJianCode/deep\_learning/tree/master/final\_homework

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\reftitle{引用}

\begin{thebibliography}{999}
% Reference 1
\bibitem{d2l}
张 Zhang, Aston,李沐,美 立顿 Lipton, Zachary C,et al.动手学深度学习[M].人民邮电出版社,2019.
\bibitem{2023-china-ai-wp}
戴琼海.2023中国人工智能系列白皮书--深度学习.中国人工智能学会,2023.
\bibitem{dl_thoery}
蒙祖强，欧元汉.深度学习理论与应用:高等学校计算机专业系列教材[M].北京:清华大学出版社,2023.
\bibitem{vgg}
Simonyan K, Zisserman A. Very deep convolutional networks for large-scale image recognition[J]. arXiv preprint arXiv:1409.1556, 2014.
\end{thebibliography}
\end{document}

